{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tight Integration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.1.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib import keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensorboard Integration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.datasets import cifar100\n",
    "\n",
    "(X_train, Y_train), (X_test, Y_test) = cifar100.load_data(label_mode='fine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "\n",
    "img_rows, img_cols = 32, 32\n",
    "\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    shape_ord = (3, img_rows, img_cols)\n",
    "else:  # channel_last\n",
    "    shape_ord = (img_rows, img_cols, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32, 32, 3)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shape_ord"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 32, 32, 3)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "nb_classes = len(np.unique(Y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.applications import vgg16\n",
    "from keras.layers import Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 32, 32, 3)         0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 32, 32, 64)        1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 32, 32, 64)        36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 16, 16, 128)       73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 16, 16, 128)       147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 8, 8, 256)         295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 8, 8, 256)         590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 8, 8, 256)         590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 4, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 4, 4, 512)         1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 4, 4, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 4, 4, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 2, 2, 512)         0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 2, 2, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 2, 2, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 2, 2, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 1, 1, 512)         0         \n",
      "=================================================================\n",
      "Total params: 14,714,688\n",
      "Trainable params: 14,714,688\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "vgg16_model = vgg16.VGG16(weights='imagenet', include_top=False, \n",
    "                          input_tensor=Input(shape_ord))\n",
    "vgg16_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for layer in vgg16_model.layers:\n",
    "    layer.trainable = False  # freeze layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.layers.core import Dense, Dropout, Flatten\n",
    "from keras.layers.normalization import BatchNormalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = Flatten(input_shape=vgg16_model.output.shape)(vgg16_model.output)\n",
    "x = Dense(4096, activation='relu', name='ft_fc1')(x)\n",
    "x = Dropout(0.5)(x)\n",
    "x = BatchNormalization()(x)\n",
    "predictions = Dense(nb_classes, activation = 'softmax')(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#create graph of your new model\n",
    "model = Model(inputs=vgg16_model.input, outputs=predictions)\n",
    "\n",
    "#compile the model\n",
    "model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 32, 32, 3)         0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 32, 32, 64)        1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 32, 32, 64)        36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 16, 16, 128)       73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 16, 16, 128)       147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 8, 8, 256)         295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 8, 8, 256)         590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 8, 8, 256)         590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 4, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 4, 4, 512)         1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 4, 4, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 4, 4, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 2, 2, 512)         0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 2, 2, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 2, 2, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 2, 2, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 1, 1, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "ft_fc1 (Dense)               (None, 4096)              2101248   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 4096)              0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 4096)              16384     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 100)               409700    \n",
      "=================================================================\n",
      "Total params: 17,242,020\n",
      "Trainable params: 2,519,140\n",
      "Non-trainable params: 14,722,880\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `TensorBoard` Callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.callbacks import TensorBoard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "\n",
    "# Arguments\n",
    "    log_dir: the path of the directory where to save the log\n",
    "        files to be parsed by TensorBoard.\n",
    "    histogram_freq: frequency (in epochs) at which to compute activation\n",
    "        and weight histograms for the layers of the model. If set to 0,\n",
    "        histograms won't be computed. Validation data (or split) must be\n",
    "        specified for histogram visualizations.\n",
    "    write_graph: whether to visualize the graph in TensorBoard.\n",
    "        The log file can become quite large when\n",
    "        write_graph is set to True.\n",
    "    write_grads: whether to visualize gradient histograms in TensorBoard.\n",
    "        `histogram_freq` must be greater than 0.\n",
    "    write_images: whether to write model weights to visualize as\n",
    "        image in TensorBoard.\n",
    "    embeddings_freq: frequency (in epochs) at which selected embedding\n",
    "        layers will be saved.\n",
    "    embeddings_layer_names: a list of names of layers to keep eye on. If\n",
    "        None or empty list all the embedding layer will be watched.\n",
    "    embeddings_metadata: a dictionary which maps layer name to a file name\n",
    "        in which metadata for this embedding layer is saved. \n",
    "```\n",
    "\n",
    "See the [details](https://www.tensorflow.org/how_tos/embedding_viz/#metadata_optional)\n",
    "about metadata files format. In case if the same metadata file is used for all embedding layers, string can be passed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 1)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## one-hot Encoding of labels (1 to 100 classes)\n",
    "from keras.utils import np_utils\n",
    "Y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y_train = np_utils.to_categorical(Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 100)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Summary name block1_conv1/kernel:0 is illegal; using block1_conv1/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name block1_conv1/bias:0 is illegal; using block1_conv1/bias_0 instead.\n",
      "INFO:tensorflow:Summary name block1_conv2/kernel:0 is illegal; using block1_conv2/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name block1_conv2/bias:0 is illegal; using block1_conv2/bias_0 instead.\n",
      "INFO:tensorflow:Summary name block2_conv1/kernel:0 is illegal; using block2_conv1/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name block2_conv1/bias:0 is illegal; using block2_conv1/bias_0 instead.\n",
      "INFO:tensorflow:Summary name block2_conv2/kernel:0 is illegal; using block2_conv2/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name block2_conv2/bias:0 is illegal; using block2_conv2/bias_0 instead.\n",
      "INFO:tensorflow:Summary name block3_conv1/kernel:0 is illegal; using block3_conv1/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name block3_conv1/bias:0 is illegal; using block3_conv1/bias_0 instead.\n",
      "INFO:tensorflow:Summary name block3_conv2/kernel:0 is illegal; using block3_conv2/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name block3_conv2/bias:0 is illegal; using block3_conv2/bias_0 instead.\n",
      "INFO:tensorflow:Summary name block3_conv3/kernel:0 is illegal; using block3_conv3/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name block3_conv3/bias:0 is illegal; using block3_conv3/bias_0 instead.\n",
      "INFO:tensorflow:Summary name block4_conv1/kernel:0 is illegal; using block4_conv1/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name block4_conv1/bias:0 is illegal; using block4_conv1/bias_0 instead.\n",
      "INFO:tensorflow:Summary name block4_conv2/kernel:0 is illegal; using block4_conv2/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name block4_conv2/bias:0 is illegal; using block4_conv2/bias_0 instead.\n",
      "INFO:tensorflow:Summary name block4_conv3/kernel:0 is illegal; using block4_conv3/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name block4_conv3/bias:0 is illegal; using block4_conv3/bias_0 instead.\n",
      "INFO:tensorflow:Summary name block5_conv1/kernel:0 is illegal; using block5_conv1/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name block5_conv1/bias:0 is illegal; using block5_conv1/bias_0 instead.\n",
      "INFO:tensorflow:Summary name block5_conv2/kernel:0 is illegal; using block5_conv2/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name block5_conv2/bias:0 is illegal; using block5_conv2/bias_0 instead.\n",
      "INFO:tensorflow:Summary name block5_conv3/kernel:0 is illegal; using block5_conv3/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name block5_conv3/bias:0 is illegal; using block5_conv3/bias_0 instead.\n",
      "INFO:tensorflow:Summary name ft_fc1/kernel:0 is illegal; using ft_fc1/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name ft_fc1/bias:0 is illegal; using ft_fc1/bias_0 instead.\n",
      "INFO:tensorflow:Summary name batch_normalization_1/gamma:0 is illegal; using batch_normalization_1/gamma_0 instead.\n",
      "INFO:tensorflow:Summary name batch_normalization_1/beta:0 is illegal; using batch_normalization_1/beta_0 instead.\n",
      "INFO:tensorflow:Summary name batch_normalization_1/moving_mean:0 is illegal; using batch_normalization_1/moving_mean_0 instead.\n",
      "INFO:tensorflow:Summary name batch_normalization_1/moving_variance:0 is illegal; using batch_normalization_1/moving_variance_0 instead.\n",
      "INFO:tensorflow:Summary name dense_1/kernel:0 is illegal; using dense_1/kernel_0 instead.\n",
      "INFO:tensorflow:Summary name dense_1/bias:0 is illegal; using dense_1/bias_0 instead.\n",
      "Epoch 1/20\n",
      "781/781 [==============================] - 49s - loss: 0.0161 - acc: 0.9974    \n",
      "Epoch 2/20\n",
      "781/781 [==============================] - 48s - loss: 1.1923e-07 - acc: 1.0000    \n",
      "Epoch 3/20\n",
      "781/781 [==============================] - 47s - loss: 1.1922e-07 - acc: 1.0000    - ETA:\n",
      "Epoch 4/20\n",
      "781/781 [==============================] - 47s - loss: 1.1922e-07 - acc: 1.0000    \n",
      "Epoch 5/20\n",
      "781/781 [==============================] - 48s - loss: 1.1922e-07 - acc: 1.0000    \n",
      "Epoch 6/20\n",
      "781/781 [==============================] - 48s - loss: 1.1921e-07 - acc: 1.0000    \n",
      "Epoch 7/20\n",
      "781/781 [==============================] - 47s - loss: 1.1921e-07 - acc: 1.0000    \n",
      "Epoch 8/20\n",
      "781/781 [==============================] - 48s - loss: 1.1922e-07 - acc: 1.0000    \n",
      "Epoch 9/20\n",
      "781/781 [==============================] - 48s - loss: 1.1921e-07 - acc: 1.0000    \n",
      "Epoch 10/20\n",
      "781/781 [==============================] - 47s - loss: 1.1921e-07 - acc: 1.0000    - ET\n",
      "Epoch 11/20\n",
      "781/781 [==============================] - 48s - loss: 1.1921e-07 - acc: 1.0000    \n",
      "Epoch 12/20\n",
      "781/781 [==============================] - 47s - loss: 1.1921e-07 - acc: 1.0000    \n",
      "Epoch 13/20\n",
      "781/781 [==============================] - 47s - loss: 1.1921e-07 - acc: 1.0000    \n",
      "Epoch 14/20\n",
      "781/781 [==============================] - 48s - loss: 1.1921e-07 - acc: 1.0000    \n",
      "Epoch 15/20\n",
      "781/781 [==============================] - 46s - loss: 1.1921e-07 - acc: 1.0000    - ETA: 0s - loss: 1.1921e-07 - acc:\n",
      "Epoch 16/20\n",
      "781/781 [==============================] - 47s - loss: 1.1921e-07 - acc: 1.0000    \n",
      "Epoch 17/20\n",
      "781/781 [==============================] - ETA: 0s - loss: 1.1921e-07 - acc: 1.000 - 47s - loss: 1.1921e-07 - acc: 1.0000    \n",
      "Epoch 18/20\n",
      "781/781 [==============================] - 47s - loss: 1.1921e-07 - acc: 1.0000    \n",
      "Epoch 19/20\n",
      "781/781 [==============================] - 47s - loss: 1.1921e-07 - acc: 1.0000    \n",
      "Epoch 20/20\n",
      "781/781 [==============================] - 47s - loss: 1.1921e-07 - acc: 1.0000    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fdb8f8f2be0>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def generate_batches(X, Y, batch_size=128):\n",
    "    \"\"\"\"\"\"\n",
    "    # Iterations has to go indefinitely\n",
    "    start = 0\n",
    "    while True:\n",
    "        yield (X[start:start+batch_size], Y[start:start+batch_size])\n",
    "        start=batch_size\n",
    "\n",
    "batch_size = 64\n",
    "steps_per_epoch = np.floor(X_train.shape[0] / batch_size)\n",
    "model.fit_generator(generate_batches(X_train, Y_train, batch_size=batch_size),\n",
    "                    steps_per_epoch=steps_per_epoch, epochs=20, verbose=1, \n",
    "                    callbacks=[TensorBoard(log_dir='./tf_logs', histogram_freq=10, \n",
    "                                           write_graph=True, write_images=True, \n",
    "                                           embeddings_freq=10, \n",
    "                                           embeddings_layer_names=['block1_conv2', \n",
    "                                                                   'block5_conv1', \n",
    "                                                                   'ft_fc1'], \n",
    "                                           embeddings_metadata=None)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Runing Tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "python -m tensorflow.tensorboard --logdir=./tf_logs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `tf.Queue` integration with `Keras`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Source**: [https://gist.github.com/Dref360/43e20eda5eb5834b61bc06a4c1855b29](https://gist.github.com/Dref360/43e20eda5eb5834b61bc06a4c1855b29)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import operator\n",
    "import threading\n",
    "from functools import reduce\n",
    "\n",
    "import keras\n",
    "import keras.backend as K\n",
    "from keras.engine import Model\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import time\n",
    "from keras.layers import Conv2D\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prod(factors):\n",
    "    return reduce(operator.mul, factors, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TRAINING = True\n",
    "with K.get_session() as sess:\n",
    "    shp = [10, 200, 200, 3]\n",
    "    shp1 = [10, 7, 7, 80]\n",
    "    inp = K.placeholder(shp)\n",
    "    inp1 = K.placeholder(shp1)\n",
    "    queue = tf.FIFOQueue(20, [tf.float32, tf.float32], [shp, shp1])\n",
    "    x1, y1 = queue.dequeue()\n",
    "    enqueue = queue.enqueue([inp, inp1])\n",
    "    model = keras.applications.ResNet50(False, \"imagenet\", x1, shp[1:])\n",
    "    for i in range(3):\n",
    "        model.layers.pop()\n",
    "        model.layers[-1].outbound_nodes = []\n",
    "        model.outputs = [model.layers[-1].output]\n",
    "    output = model.outputs[0]  # 7x7\n",
    "    # Reduce filter size to avoid OOM\n",
    "    output = Conv2D(32, (1, 1), padding=\"same\", activation='relu')(output)\n",
    "    output3 = Conv2D(5 * (4 + 11 + 1), (1, 1), padding=\"same\", activation='relu')(\n",
    "        output)  # YOLO output B (4 + nb_class +1)\n",
    "    cost = tf.reduce_sum(tf.abs(output3 - y1))\n",
    "    optimizer = tf.train.RMSPropOptimizer(0.001).minimize(cost)\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "\n",
    "    def get_input():\n",
    "        # Super long processing I/O bla bla bla\n",
    "        return np.arange(prod(shp)).reshape(shp).astype(np.float32), np.arange(prod(shp1)).reshape(shp1).astype(\n",
    "            np.float32)\n",
    "\n",
    "\n",
    "    def generate(coord, enqueue_op):\n",
    "        while not coord.should_stop():\n",
    "            inp_feed, inp1_feed = get_input()\n",
    "            sess.run(enqueue_op, feed_dict={inp: inp_feed, inp1: inp1_feed})\n",
    "\n",
    "\n",
    "    start = time.time()\n",
    "    for i in tqdm(range(10)):  # EPOCH\n",
    "        for j in range(30):  # Batch\n",
    "            x,y = get_input()\n",
    "            optimizer_, s = sess.run([optimizer, queue.size()], \n",
    "                                     feed_dict={x1:x,y1:y, K.learning_phase(): int(TRAINING)})\n",
    "    print(\"Took : \", time.time() - start)\n",
    "\n",
    "\n",
    "    coordinator = tf.train.Coordinator()\n",
    "    threads = [threading.Thread(target=generate, args=(coordinator, enqueue)) for i in range(10)]\n",
    "    for t in threads:\n",
    "        t.start()\n",
    "    start = time.time()\n",
    "    for i in tqdm(range(10)):  # EPOCH\n",
    "        for j in range(30):  # Batch\n",
    "            optimizer_, s = sess.run([optimizer, queue.size()], \n",
    "                                     feed_dict={K.learning_phase(): int(TRAINING)})\n",
    "    print(\"Took : \", time.time() - start)\n",
    "\n",
    "    def clear_queue(queue, threads):\n",
    "        while any([t.is_alive() for t in threads]):\n",
    "            _, s = sess.run([queue.dequeue(), queue.size()])\n",
    "            print(s)\n",
    "\n",
    "\n",
    "    coordinator.request_stop()\n",
    "    clear_queue(queue, threads)\n",
    "\n",
    "    coordinator.join(threads)\n",
    "    print(\"DONE Queue\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
